<?php 
	/**
	 * 自动注册缘来id
	 */
	function registerYuanlaiId()
	{
		//step1: get Yuanlai view cookie
		$url = "http://www.yuanlai.com/register/registered.do";
		$page = makeRequest($url,
						 "GET",
						 "",
						 true,
						 ROOT_PATH."/yuanlai/cookie/yuanlai_view_cookie.txt",
						 "",  
						 ""
						 );
		
		//step2: auto register
		$regInfo  = array();
		$url = "http://www.yuanlai.com/register/registered.do?step=regok";
		$base = substr(strval(microtime(true)), -8);
		$regdata = createRegData();
	
		$parameter = array("uu.email" => $regdata['email'],
							"uu.password" => "123abc",
							"uu.nickname" => createRandStr(),
							"dateForm.year" => "1984",
							"dateForm.month"=>"3",
							"dateForm.day"=>"15",
							"uu.marriage"=>"1",
							"uu.height"=>"175",
							"uu.education"=>"4",
							"uu.province"=>"10124000",
							"uu.workCity"=>"10102000",
							"uu.salary"=>"6",
							"uu.gender" => "1",
							"forward"=>"/",
							);
		$opt_arr = array(CURLOPT_REFERER => "http://www.yuanlai.com/register/registered.do",  //referer
			//			 CURLOPT_HEADER => true,						//输出中包含头信息
			//			 CURLOPT_COOKIE => $cookie,					
					);
		printlog("using:".$parameter['uu.email']." as register email", "register");
		$page = makeRequest($url,
						 "post",
						 $parameter,
						 true, 
						 ROOT_PATH."/yuanlai/cookie/yuanlai_login_cookie.txt",  
						 ROOT_PATH."/yuanlai/cookie/yuanlai_view_cookie.txt",
						 $opt_arr
						 );
		//file_put_contents("test.html", $page);
		//exit;
						 
		//step3: parse uid from page
		preg_match("%ID:([\d]*)</p%i", $page, $arr);
		return $arr[1];
	}
	/*
	*生成注册数据。
	*/
	function createRegData ()
	{
		$mail_array= array('@126.com','@qq.com','@163.com','@gmail.com','@tom.com','@yeah.net','@live.cn');
		$data['email'] = createRandStr().'_girl'.$mail_array[array_rand($mail_array)];
		return $data;
	}
	/*
	*生成随机的字符
	*/
	function createRandStr ()
	{	
		$s = '';
		$str = "0123456789abcdefghijklmnopqrstuvwxyz";   //   输出字符集  
		$n = 10;   //   输出串长度  
		$len = strlen($str)-1;
		for($i=0 ; $i<$n; $i++){
			$s .=  $str[rand(0,$len)];  
		}
		
		return $s;
	}
	//自动保存新注册的id
	function saveYuanlaiId($new_Yuanlai_id)
	{
		global $conn;
		$today = date("Y-m-d", time());
		$now = date("Y-m-d H:i:s", time());
		$info = $now."\t".$new_Yuanlai_id."\n";
		//保存至文件（防止数据库挂掉，备查）
		file_put_contents(ROOT_PATH."/yuanlai/log/register_log/$today.log", $info, FILE_APPEND);
		//保存至数据库
		$arr = array("site" => "yuanlai",
					"ymd" => $today,
					"register_time" => $now,
					"source_user_id" => $new_Yuanlai_id
				);
		$sql = Sql::insert($arr, "register_log");
		$flag = $conn->query($sql);
		return $flag;
	}
	
	//获取某一天内缘来最小的id
	function getYuanlaiMinId($ymd)
	{
		global $conn;
		$sql = "select min(source_user_id) from register_log where ymd='$ymd' and site='yuanlai'";
		$minId = $conn->result($sql);
		return $minId;
	}
	
	//获取某一天内缘来最大的id
	function getYuanlaiMaxId($ymd)
	{
		global $conn;
		$sql = "select max(source_user_id) from register_log where ymd='$ymd' and site='yuanlai'";
		$maxId = $conn->result($sql);
		return $maxId;
	}
	
	//获取某一天内缘来已经抓到的最大id
	function getYuanlaiMaxIdFinished($ymd)
	{
		global $conn;
		$tblName = getTblName($ymd);
		$sql = "select max(source_user_id) from $tblName where fetch_date='$ymd' and `from`='yuanlai'";//echo $sql;exit;
		$maxIdFinished = $conn->result($sql);
		return $maxIdFinished;
	}
	
	//获取缘来某一个id的注册时间，debug时候用
	function getYuanlaiRegTime($uid)
	{
		global $conn;
		$sql = "select register_time from register_log where source_user_id='$uid' and site='Yuanlai'";
		$regTime = $conn->result($sql);
		return $regTime;
	}
	
	//判断缘来抓下来的页面是否包含正常的uid（通过经验判断，审核不通过的uid页面会非常小）
	function checkYuanlaiIdExists($page)
	{
		if(strlen($page)<130)
			return false;
		else
			return true;
	}
	
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理缘来业务，包括抓取、提取和入库
	function yuanlai_crawlbak($startId, $endId, $ymd)
	{
		//创建page目录
		$page_dir = ROOT_PATH."/yuanlai/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$continueMiss = 0; //连续id不存在的个数
		$cnt_succ = 0; //id存在的总个数
		for ($uid=$startId; $uid<$endId; $uid++){	
			$page = "";	
			//避免断线情况，这里采用循环抓取，直到抓到页面为止
			do{
				$url = "http://www.yuanlai.com/profile/index.do?memberId=".$uid;
				//printlog("start crawl url : $url");
				$page = makeRequest($url, 
								"GET" , 
								"", 
								false, 
								"", 
								ROOT_PATH."/Yuanlai/cookie/Yuanlai_login_cookie.txt",
								"");
				//echo strlen($page)."\n"; 
			} while($page === false);
		
			if(checkYuanlaiIdExists($page)){
				printlog("$uid available", "crawl");
				if(file_put_contents($page_dir.$uid.".html", $page)){
					doWithYuanlaiPage($uid, $ymd, $page);
				}
				else{
					printlog("save page error", "crawl");
				}
				$cnt_succ++;
				$continueMiss = 0;
			}else{
				printlog("$uid not available", "crawl");
				$continueMiss ++ ;
			}
	
			//防止出现id跳跃的现象，如果连续50个不存在，则以间隔50向前跳，直到抓取到一个id为止再恢复间隔为1
			if($continueMiss > 50){
				$uid += 50;
			}
		}
	}
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理珍爱业务，包括抓取、提取和入库
	function yuanlai_crawl($startId, $endId, $ymd)
	{
		//创建page目录
		$page_dir = ROOT_PATH."/yuanlai/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		//如果抓取的速度不够，改用multi
		$number = 30;//concurrent request number
		$continueMiss = 0;
		$previousExists = true;
		$previousId = $startId;
		for ($uid=$startId; $uid<$endId; ){
			//保留本次开始的uid，万一抓取失败了重新抓取
			$previousId = $uid; 
			$perRequestFailedCount  = 0 ;//记录本次抓取失败的个数
			$handles = makeYuanlaiMultiRequests($uid, $number);
			foreach($handles as $thisuid => $handle){
				$page = curl_multi_getcontent($handle);
				if($page === false){ 
					printlog ("curl_multi_getcontent error: ".curl_error($handle), "crawl");
				}
				
				if(checkYuanlaiIdExists($page)){
					printlog("$thisuid available", "crawl");
					if(file_put_contents($page_dir.$thisuid.".html", $page)){	
						doWithYuanlaiPage($thisuid, $ymd, $page);
					}
					else{
						printlog("save page error", "crawl");
					}
					$continueMiss = 0;
				}else{
					if(strlen($page)<30){
						echo "network problem ret is ".$page;
						$perRequestFailedCount++;
					}
					else{ 
						printlog("$thisuid not available", "crawl");
					}
					$continueMiss ++;
				}
				curl_close($handle);
				$uid++;
			}
				
			if($perRequestFailedCount>= $number/2){//如果有一半不存在，说明中途网络有问题了 
				$uid = $previousId;
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，重新来。。。", "crawl");
				continue;//重新来
			}
			else{
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，继续下一次抓取", "crawl");
			}
		}
	} 
	/*
	*一下抓取多个页面
	*/
	function makeYuanlaiMultiRequests($uid, $number)
	{
		$mhandle = curl_multi_init();
		$handles = array();
		while($number>0){
			$url = "http://www.yuanlai.com/profile/index.do?memberId=".$uid;
			$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
			$handles[$uid] = makeRequestResource($url,
												"GET" , 
												"", 
												false, 
												"", 
												ROOT_PATH."/yuanlai/cookie/yuanlai_login_cookie.txt", 
												$opt_arr
												);
			$uid++;
			$number--;
		}
		foreach($handles as $handle)
			curl_multi_add_handle($mhandle, $handle);
		//wait all request finish
		do {
			$status = curl_multi_exec($mhandle, $active);
		} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
		curl_multi_close($mhandle);
		
		return $handles;
	}
	//生成缘来登录后的cookie
	function makeYuanlaiCookie()
	{
		//step1: browser the mainpage and get jsession id ,dump it to Yuanlai_portal_cookie.txt	
		makeRequest("http://www.yuanlai.com", 
					"GET", 
					"", 
					true, 
					ROOT_PATH."/yuanlai/cookie/yuanlai_portal_cookie.txt",
					"",
					""
					);
		//step2 login with the cookies which is fetched by previous request
		$parameter = array("_"=>time(), 
							"rememberMe"=>"true", 
							"type"=>'page', 
							"forward"=>'/personal/myindex.do',
							"user"=>"smarteng@qq.com", 
							"password"=>"789852"
							);
		makeRequest("http://www.yuanlai.com/login/user-login.do",
					"GET", 
					$parameter, 
					true, 
					ROOT_PATH."/yuanlai/cookie/yuanlai_login_cookie.txt", 
					ROOT_PATH."/yuanlai/cookie/yuanlai_portal_cookie.txt",
					""
					);
	}
		
	//对一个抓下的缘来页面的后续处理
	function doWithYuanlaiPage($uid, $ymd, $page)
	{
		//check uid exists?
		global $conn, $global_province_list;
		$tblName = getTblName();
		$sql = "select * from $tblName where `from`='yuanlai' and source_user_id='$uid' limit 1";
		$exists = $conn->has_record($sql);
		if ($exists){
			printlog("$uid exists, do nothing", "crawl");
			return 1;
		}
		else{
			//extract
			//tmp: 山东济南34岁 或 江苏省20岁
			$tmp = simpleExtract('%profile_intro_row"><strong>(.*?)</strong>%i', $page, "tmp");
			$tmp = strip_tags($tmp);
			echo $tmp."<br />";
			
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($tmp, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			$age = simpleExtract('/(\d\d)/', $tmp, "age"); //20
			$city = simpleExtract('/来自(.*)?/', $tmp, "city"); //20
			$city = str_replace(array($province, $age, "省"), "", $city); //济南
			
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			$gender = simpleExtract('/(.*)?士/', $tmp, "gender");
			$gender = ($gender=="男")?1:0; //男为1 女为0
			
			preg_match('/<li.*?class="first-child".*?><img.*?src="(.*?)"/i', $page, $matches);
						
			$avat_array = array('300nv.png','300nj.png');
			if(isset($matches[1])){
				$photoarr = explode( '/' ,$matches[1]);
				if(is_array($photoarr)){
					$potofile = $photoarr[count($photoarr)-1];
					$has_photo = in_array($potofile,$avat_array)?0:1;
				}else{
					$has_photo = 0 ;
				}
			}else{
				preg_match('/<li.*?><img.*?name="profile_img".*?src="(.*?)"/i', $page, $match);
				if(isset($match[1])){
					$photoarray = explode( '/' ,$match[1]);
					if(is_array($photoarray)){
						$imgfile = $photoarray[count($photoarray)-1];
						$has_photo = in_array($imgfile ,$avat_array)?0:1;
					}else{
						$has_photo = 0 ;
					}
				}
			}
			
			//insert
			$arr = array("fetch_date" => $ymd,
					"`from`" => "yuanlai",
					"source_user_id" => $uid,
					"age" => $age,
					"province" => $province,
					"city" => $city,
					"gender" => $gender,
					"has_photo" => $has_photo,
				);
			$sql = Sql::insert($arr, $tblName);
			$conn->query($sql);
		}
	}
	//对一个抓下的缘来页面的后续处理
	function doWithYuanlai($page)
	{
		//check uid exists?
		global $conn, $global_province_list;
			//extract
			//tmp: 山东济南34岁 或 江苏省20岁
			$tmp = simpleExtract('%profile_intro_row"><strong>(.*?)</strong>%i', $page, "tmp");
			$tmp = strip_tags($tmp);
			echo $tmp."<br />";
			
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($tmp, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			$age = simpleExtract('/(\d\d)/', $tmp, "age"); //20
			$city = simpleExtract('/来自(.*)?/', $tmp, "city"); //20
			$city = str_replace(array($province, $age, "省"), "", $city); //济南
			
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			$gender = simpleExtract('/(.*)?士/', $tmp, "gender");
			$gender = ($gender=="男")?1:0; //男为1 女为0
			
			preg_match('/<li.*?class="first-child".*?><img.*?src="(.*?)"/i', $page, $matches);
						
			$avat_array = array('300nv.png','300nj.png');
			if(isset($matches[1])){
				$photoarr = explode( '/' ,$matches[1]);
				if(is_array($photoarr)){
					$potofile = $photoarr[count($photoarr)-1];
					$has_photo = in_array($potofile,$avat_array)?0:1;
				}else{
					$has_photo = 0 ;
				}
			}else{
				preg_match('/<li.*?><img.*?name="profile_img".*?src="(.*?)"/i', $page, $match);
				if(isset($match[1])){
					$photoarray = explode( '/' ,$match[1]);
					if(is_array($photoarray)){
						$imgfile = $photoarray[count($photoarray)-1];
						$has_photo = in_array($imgfile ,$avat_array)?0:1;
					}else{
						$has_photo = 0 ;
					}
				}
			}
			echo "has_photo:".$has_photo."\n";
		
	}

?>